# -*- coding:utf-8 -*-
from pyspark import SparkConf,SparkContext
from pathlib import Path
from urllib import request
import json
from urllib import parse
import re
base_url = "http://nanning.gongjiao.com"
lines_num = ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'w', 'x', 'y', 'z']
# lines_num = ['1']
conf = SparkConf().setMaster("local[4]").setAppName("SparkDemo")
sc = SparkContext(conf=conf)


"""
初始化线路存储目录
"""
def init_lines():
    p = Path('./lines')
    if p.exists() == False:
        p.mkdir()
"""
下载html
"""
def download_html(url):

    req = request.Request(url)
    req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
    with request.urlopen(req) as f:
        data = f.read().decode("utf-8")
        return data

"""
下载线路号信息
"""
def download_line(num):
    url = base_url+"/lines_%s.html" % num
    print("url:%s" % url)
    data = download_html(url)
    return parse_line(num,data)

"""
解析线路查询页
"""
def parse_line(num,data):
    res = re.findall('<div class="list">(.*?)</div>',data,re.S)
    ress = []
    if len(res)==1:
        data = res[0]
        res = re.findall('<a href="http://nanning.gongjiao.com/xianlu_([0-9]*)">(.*?)</a>',data,re.S)

        for item in res:
            ress.append((num,item[0],item[1]))
        return ress
    else:
        return ress

"""
解析线路
"""
def parse_xianlu(data):
    ress = []
    res = re.findall('<ul class="gj01_line_img JS-up clearfix">(.*?)</ul>',data,re.S)

    if len(res)!=0:
        data = res[0]
        res = re.findall('<a target="_blank" href="http://[0-9a-z]*?.gongjiao.com/zhandian_([0-9]*)"><i></i>(.*?)</a>',data,re.S)
        if len(res)!=0:
            return res
    return ress

"""
下载并解析线路详情
"""
def download_and_parse_xianlu(item):
    url = base_url+"/xianlu_%s" % item[1]
    print("url:%s" % url)
    data = download_html(url)
    res = parse_xianlu(data)
    ress = []
    for i in res:
        ress.append(item+i)
    return ress

"""
地址转经纬度
"""
def address_to_location(item):
    ress = ""
    city = parse.quote("南宁")
    address = parse.quote(item['zhandian'])
    url = "https://restapi.amap.com/v3/geocode/geo?key=667dd97cc9ca502c88dee11bfdabf056&=city=%s&address=%s&output=JSON" % (city,address)
    res = download_html(url)
    print(res)
    data = json.loads(res)
    if data['status'] == '1':
        if len(data['geocodes'])!=0:
            ress = data['geocodes'][0]['location']
    item['location'] = ress
    return item

if __name__== "__main__":
    res = sc.parallelize((lines_num)).map(download_line)
    filterRes = res.filter(lambda item: len(item)>0).flatMap(lambda item:item).map(download_and_parse_xianlu)\
        .flatMap(lambda item:item)\
        .map(lambda item: {"id":item[3],"searchId":item[0],"lineId":item[1],"lineName":item[2],"zhandian":item[4]})\
        .map(address_to_location)\
        .collect()
    for item in filterRes:
        print(item)
    # a = ('8', '27294', '南宁89路', ('234040', '朝阳广场(文化宫)'), ('233736', '友爱民乐路口'), ('233748', '友爱民生路口'), ('234021', '南环路'), ('233530', '市一医院'), ('233349', '新民七星路口'), ('233500', '区医院'), ('233499', '桃源路中'), ('233733', '桃源双拥路口'), ('233498', '医科大一附院'), ('233497', '双拥路中'), ('233496', '医科大学'), ('233494', '南宁三中'), ('233493', '青山路中'), ('234064', '竹溪青山路口'), ('233954', '会展路南'), ('233932', '越秀会展路口'), ('233933', '越秀路小学'), ('233934', '越秀路中'), ('233624', '桂雅青秀路口'), ('233610', '东盟公交站'), ('234066', '桂雅路中'), ('234067', '桂雅合作路口'), ('233850', '翡翠园北门'), ('328891', '青林青环路口(埌东客运站)'), ('233833', '仙葫大道路口'), ('233834', '高速公路管理处'), ('233835', '西区路口'), ('233836', '上水人家'), ('233837', '仙葫盘古路口'), ('234068', '开泰仙葫路口'), ('234069', '通福开泰路口'), ('234070', '通福蓉茉路口'), ('234041', '仙葫管委会'), ('234071', '东方广场'), ('234071', '东方广场'), ('234041', '仙葫管委会'), ('234070', '通福蓉茉路口'), ('234072', '开泰通福路口'), ('233838', '仙葫开泰路口'), ('233837', '仙葫盘古路口'), ('233836', '上水人家'), ('233835', '西区路口'), ('233834', '高速公路管理处'), ('233833', '仙葫大道路口'), ('328891', '青林青环路口(埌东客运站)'), ('233850', '翡翠园北门'), ('234067', '桂雅合作路口'), ('234066', '桂雅路中'), ('233610', '东盟公交站'), ('233874', '越秀青秀路口'), ('233934', '越秀路中'), ('233933', '越秀路小学'), ('234083', '会展越秀路口'), ('233954', '会展路南'), ('233490', '青秀山大门'), ('233492', '青山菜市'), ('233493', '青山路中'), ('233494', '南宁三中'), ('233496', '医科大学'), ('233497', '双拥路中'), ('233498', '医科大一附院'), ('233733', '桃源双拥路口'), ('233499', '桃源路中'), ('233500', '区医院'), ('233531', '七星新民路口'), ('233530', '市一医院'), ('234021', '南环路'), ('233748', '友爱民生路口'), ('233736', '友爱民乐路口'), ('234040', '朝阳广场(文化宫)'))
    # print(a[0:3]+a[3])
    sc.parallelize(filterRes).coalesce(1).saveAsTextFile("file:///D:/mldong/bigdata/公交站点经纬度")

